Memory Magic Mystery

Hi All,

I'm chasing some memory leaks using a hacked up version of Mathias Mamsch's memory leak magic DXL from:
https://www.ibm.com/developerworks/forums/thread.jspa?messageID=14832712&#14832712

and have stumbled into a bit of a mystery I thought had to do with dialog boxes. Turns out the first call to delete or destroy is creating a new object (entry on the allocations list).

If anyone has an explanation or thoughts on what is going on I'd appreciate it. But don't spend to much time thinking about it. It's just a curiosity.

Here is my test code:

void printHexNibble( int nibble )
{
   print charOf( nibble + (nibble < 10 ? intOf('0') : intOf('A') - 10 ) )
}

void printHexByte( int byte )
{
   printHexNibble( ( byte & 0xF0 ) / 16 )
   printHexNibble( byte & 0x0F )
}

void printHexWord( int word )
{
      printHexByte( (word / 0x1000000 ) & 0xFF )
      printHexByte( (word / 0x0010000 ) & 0xFF )
      printHexByte( (word / 0x0000100 ) & 0xFF )
      printHexByte( (word             ) & 0xFF )
}


int peek8( int * address, int offset )
{
   address += offset
   return *address & 255
}

int peek32Int( int * address, int offset )
{
   address += offset
   return (*address) | 0
}

int * peek32Ptr( int * address, int offset )
{
   int v = peek32Int( address, offset )
   int * rv = addr_( v )
   return rv
}


void printMemHexWord( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      printHexWord( peek32Int( address, i*4 ) )
      print "\n"
   }
}

void printMemCharByte( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      print charOf( peek8( address, i ) )
   }
}

void printMemHexByte( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      printHexByte( peek8( address, i ) )
   }
}


/*
 * Memory Leak Debug Magic
 */

// add offset to address

int * ::+ (int *ptr1, int ofs)
{
  int *ptr2 = ptr1
  ptr2 += ofs
  return ptr2
}

// get context pointer ???
 
int *ccp ()
{
   DB x = create ""
   int *p = addr_( x )
   int *r = peek32Ptr( p, 48 )
   destroy x
   return r
}

// Get contex pointer from temporary dialog box ???

int * contextPtr = ccp()

// Get address of allocation list head pointer ???

int * listHead = contextPtr + 0x74

// Get & print elements on the allocation list
 
int allocatedObjects()
{
   int cnt = 0

   // Get first element on allocation list
   int *mb = addr_( peek32Int( listHead, 0 ) )

   while(!null mb) 
   {
      print "LE->"
      print cnt ""
      print "->"
      printHexWord( (int addr_( mb ) ) )
      print "->"
      printHexWord( peek32Int( mb, 4 ) )
      print "\n"

      //Get next element on allocation list
      mb = peek32Ptr( mb, 8 )
      cnt++
   }

   return cnt 
}

print "Start\n"
print allocatedObjects()
print "\n"

DB boatBox = create( "Craft" )

print "DB created: "
printHexWord( (int addr_( boatBox ) ) )
print "\n"
print allocatedObjects()
print "\n"

string boats[] = {"Dinghy", "Destroyer", "Carrier", "Mine sweeper"}

Regexp r = regexp2( "hi" )

print "Regexp created: "
printHexWord( (int addr_( r ) ) )
print "\n"
print allocatedObjects()
print "\n"

Buffer b = create()

print "Buffer created: "
printHexWord( (int addr_( b ) ) )
print "\n"
print allocatedObjects()
print "\n"

Skip   s = create()

print "Skip created: "
printHexWord( (int addr_( s ) ) )
print "\n"
print allocatedObjects()
print "\n"

DBE boatCheck = radioBox(boatBox, "Select class:", boats, 3)
int * ptr = addr_( boatCheck )

print "DBE created: "
printHexWord( (int addr_( boatCheck ) ) )
print "\n"
print allocatedObjects()
print "\n"


void toBuild(DBE option)
{
    int favorite = get( option )
    ack(boatBox, "You are planning a new " boats[favorite] "?")
}

set( boatCheck, toBuild ) 

block( boatBox )

print "Dialog Box Closed\n\n"

destroy( boatBox )

print "DB destroyed:\n"
print allocatedObjects()
print "\n"

delete( r )

print "Regexp deleted:\n"
print allocatedObjects()
print "\n"

delete( b )

print "Buffer deleted:\n"
print allocatedObjects()
print "\n"

delete( s )

print "Skip deleted:\n"
print allocatedObjects()
print "\n"

Here are the results I get:

Start
0

DB created: 088B71A0
LE->0->088B0B10->088B71A0     //DB
1

Regexp created: 088B5480
LE->0->088B2CD0->088B5480     //Regexp
LE->1->088B0B10->088B71A0     //DB
2

Buffer created: 088B0B60
LE->0->088B09C0->088B0B60     //Buffer
LE->1->088B2CD0->088B5480     //Regexp
LE->2->088B0B10->088B71A0     //DB
3

Skip created: 088B5060
LE->0->088B09C0->088B0B60     //Buffer
LE->1->088B2D50->088B5060     //Skip
LE->2->088B2CD0->088B5480     //Regexp
LE->3->088B0B10->088B71A0     //DB
4

DBE created: 088B70C0
LE->0->088B09C0->088B0B60     //Buffer
LE->1->088B2D50->088B5060     //Skip
LE->2->088B2CD0->088B5480     //Regexp
LE->3->088B0B10->088B71A0     //DB
4

Dialog Box Closed

DB destroyed:
LE->0->088B2D30->088B09B0     //??
LE->1->088B09C0->088B0B60     //Buffer
LE->2->088B2D50->088B5060     //Skip
LE->3->088B2CD0->088B5480     //Regexp
4

Regexp deleted:
LE->0->088B2D30->088B09B0     //??
LE->1->088B09C0->088B0B60     //Buffer
LE->2->088B2D50->088B5060     //Skip
3

Buffer deleted:
LE->0->088B2D30->088B09B0     //??
LE->1->088B2D50->088B5060     //Skip
2

Skip deleted:
LE->0->088B2D30->088B09B0     //??
1

The first delete or destroy is removing the proper item from the "allocations list" but is adding a new one.  It doesn't seem to matter which of the objects (DB,Buffer,Skip,Regexp) is deleted first. On the first delete the new object gets created. This leaves the balance of objects off by one.

Re: Memory Magic Mystery
GothSloth - Sat Jan 14 00:43:39 EST 2017

Hmm, maybe it does have something to do with the Dialog Box. When I take the dialog box out of the test code I get expected results. Sloths can be a little slow but this one is seriously confused.

void printHexNibble( int nibble )
{
   print charOf( nibble + (nibble < 10 ? intOf('0') : intOf('A') - 10 ) )
}

void printHexByte( int byte )
{
   printHexNibble( ( byte & 0xF0 ) / 16 )
   printHexNibble( byte & 0x0F )
}

void printHexWord( int word )
{
      printHexByte( (word / 0x1000000 ) & 0xFF )
      printHexByte( (word / 0x0010000 ) & 0xFF )
      printHexByte( (word / 0x0000100 ) & 0xFF )
      printHexByte( (word             ) & 0xFF )
}


int peek8( int * address, int offset )
{
   address += offset
   return *address & 255
}

int peek32Int( int * address, int offset )
{
   address += offset
   return (*address) | 0
}

int * peek32Ptr( int * address, int offset )
{
   int v = peek32Int( address, offset )
   int * rv = addr_( v )
   return rv
}


void printMemHexWord( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      printHexWord( peek32Int( address, i*4 ) )
      print "\n"
   }
}

void printMemCharByte( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      print charOf( peek8( address, i ) )
   }
}

void printMemHexByte( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      printHexByte( peek8( address, i ) )
   }
}


/*
 * Memory Leak Debug Magic
 */

// add offset to address

int * ::+ (int *ptr1, int ofs)
{
  int *ptr2 = ptr1
  ptr2 += ofs
  return ptr2
}

// get context pointer ???
 
int *ccp ()
{
   DB x = create ""
   int *p = addr_( x )
   int *r = peek32Ptr( p, 48 )
   destroy x
   return r
}

// Get contex pointer from temporary dialog box ???

int * contextPtr = ccp()

// Get address of allocation list head pointer ???

int * listHead = contextPtr + 0x74

// Get & print elements on the allocation list
 
int allocatedObjects()
{
   int cnt = 0

   // Get first element on allocation list
   int *mb = addr_( peek32Int( listHead, 0 ) )

   while(!null mb) 
   {
      print "LE->"
      print cnt ""
      print "->"
      printHexWord( (int addr_( mb ) ) )
      print "->"
      printHexWord( peek32Int( mb, 4 ) )
      print "\n"

      //Get next element on allocation list
      mb = peek32Ptr( mb, 8 )
      cnt++
   }

   return cnt 
}

print "Start\n"
print allocatedObjects()
print "\n"


Regexp r = regexp2( "hi" )

print "Regexp created: "
printHexWord( (int addr_( r ) ) )
print "\n"
print allocatedObjects()
print "\n"

Buffer b = create()

print "Buffer created: "
printHexWord( (int addr_( b ) ) )
print "\n"
print allocatedObjects()
print "\n"

Skip   s = create()

print "Skip created: "
printHexWord( (int addr_( s ) ) )
print "\n"
print allocatedObjects()
print "\n"


delete( r )

print "Regexp deleted:\n"
print allocatedObjects()
print "\n"

delete( b )

print "Buffer deleted:\n"
print allocatedObjects()
print "\n"

delete( s )

print "Skip deleted:\n"
print allocatedObjects()
print "\n"

Here are the results that balance out:

Start
0

Regexp created: 088ADE50
LE->0->088AC890->088ADE50  //Regexp
1

Buffer created: 088AE4F0
LE->0->088AC890->088ADE50  //Regexp
LE->1->088AD740->088AE4F0  //Buffer
2

Skip created: 088AD010
LE->0->088A9C60->088AD010  //Skip
LE->1->088AC890->088ADE50  //Regexp
LE->2->088AD740->088AE4F0  //Buffer
3

Regexp deleted:
LE->0->088A9C60->088AD010  //Skip
LE->1->088AD740->088AE4F0  //Buffer
2

Buffer deleted:
LE->0->088A9C60->088AD010  //Skip
1

Skip deleted:
0                          //This looks good

Re: Memory Magic Mystery
Mathias Mamsch - Sun Jan 15 09:16:43 EST 2017

void printHexNibble( int nibble )
{
   print charOf( nibble + (nibble < 10 ? intOf('0') : intOf('A') - 10 ) )
}

void printHexByte( int byte )
{
   printHexNibble( ( byte & 0xF0 ) / 16 )
   printHexNibble( byte & 0x0F )
}

void printHexWord( int word )
{
      printHexByte( (word / 0x1000000 ) & 0xFF )
      printHexByte( (word / 0x0010000 ) & 0xFF )
      printHexByte( (word / 0x0000100 ) & 0xFF )
      printHexByte( (word             ) & 0xFF )
}


int peek8( int * address, int offset )
{
   address += offset
   return *address & 255
}

int peek32Int( int * address, int offset )
{
   address += offset
   return (*address) | 0
}

int * peek32Ptr( int * address, int offset )
{
   int v = peek32Int( address, offset )
   int * rv = addr_( v )
   return rv
}


void printMemHexWord( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      printHexWord( peek32Int( address, i*4 ) )
      print "\n"
   }
}

void printMemCharByte( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      print charOf( peek8( address, i ) )
   }
}

void printMemHexByte( int * address, int size )
{
   int i

   for( i = 0; i < size; i++ )
   {
      printHexByte( peek8( address, i ) )
   }
}


/*
 * Memory Leak Debug Magic
 */

// add offset to address

int * ::+ (int *ptr1, int ofs)
{
  int *ptr2 = ptr1
  ptr2 += ofs
  return ptr2
}

// get context pointer ???
 
int *ccp ()
{
   DB x = create ""
   int *p = addr_( x )
   int *r = peek32Ptr( p, 48 )
   destroy x
   return r
}

// Get contex pointer from temporary dialog box ???

int * contextPtr = ccp()

// Get address of allocation list head pointer ???

int * listHead = contextPtr + 0x74

// Get & print elements on the allocation list
 
int allocatedObjects()
{
   int cnt = 0

   // Get first element on allocation list
   int *mb = addr_( peek32Int( listHead, 0 ) )

   while(!null mb) 
   {
      print "LE->"
      print cnt ""
      print "->"
      printHexWord( (int addr_( mb ) ) )
      print "->"
      printHexWord( peek32Int( mb, 4 ) )
      print "\n"

      //Get next element on allocation list
      mb = peek32Ptr( mb, 8 )
      cnt++
   }

   return cnt 
}

print "Start\n"
print allocatedObjects()
print "\n"


Regexp r = regexp2( "hi" )

print "Regexp created: "
printHexWord( (int addr_( r ) ) )
print "\n"
print allocatedObjects()
print "\n"

Buffer b = create()

print "Buffer created: "
printHexWord( (int addr_( b ) ) )
print "\n"
print allocatedObjects()
print "\n"

Skip   s = create()

print "Skip created: "
printHexWord( (int addr_( s ) ) )
print "\n"
print allocatedObjects()
print "\n"


delete( r )

print "Regexp deleted:\n"
print allocatedObjects()
print "\n"

delete( b )

print "Buffer deleted:\n"
print allocatedObjects()
print "\n"

delete( s )

print "Skip deleted:\n"
print allocatedObjects()
print "\n"

Start
0

Regexp created: 088ADE50
LE->0->088AC890->088ADE50  //Regexp
1

Buffer created: 088AE4F0
LE->0->088AC890->088ADE50  //Regexp
LE->1->088AD740->088AE4F0  //Buffer
2

Skip created: 088AD010
LE->0->088A9C60->088AD010  //Skip
LE->1->088AC890->088ADE50  //Regexp
LE->2->088AD740->088AE4F0  //Buffer
3

Regexp deleted:
LE->0->088A9C60->088AD010  //Skip
LE->1->088AD740->088AE4F0  //Buffer
2

Buffer deleted:
LE->0->088A9C60->088AD010  //Skip
1

Skip deleted:
0                          //This looks good

Maybe you can remove some empty lines in your code examples, and probably you could factor some code out to a file to make this post a bit shorter.

OK. So from your code it seems, that either "set", "block" or "destroy" puts another object in the allocated objects list. Not too bad, since you will be creating not so many dialog boxes. Is there any special problem with this?

I notice, that DOORS seems to be inserting items at the start of the allocated objects list. What DOORS version are you using?

Regards, Mathias

Re: Memory Magic Mystery
GothSloth - Mon Jan 16 19:37:14 EST 2017

No problem, more curiosity ( insert bad sloth/cat joke here ). I found it is the set() that is adding another object in the allocated objects list, good call there. The object referenced from the allocation list contains the callback function address in the second word. It looks like DOORS is not cleaning these up with the dialog box destroy(). My initial trouble was with a small data set the important memory leaks I was after were getting lost in the noise from the dialog boxes. With a much larger and slower data set I was able to find the leaks that matter and separate them from the noise.

For kicks and giggles I cast the addresses from the lingering allocations to Buffers and called delete on them. It did clean up the allocations from the list but I don't image it left memory in a very good state. Any thoughts on what those allocation blocks maybe or if there is a safe way to clean them up. Again more curiosity than any actual need. It might be handy to a cleaner function for the dialog boxes to reduce the noise when looking for leaks. I certainly would not leave it in for production code.

Code attached this time ;) Results are below.

Thanks

I'm running DOORS 9.5.1.2 client and server on Windows 10.

ContextPtr: 08B507F0

Start
Allocation List (cnt->list element address->object address):
Allocated Objects: 0

DB created at: 08B50550

Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
Allocated Objects: 1

toBuild callback function address: 0C276BC8

boatCheck DBE created at: 08B50DC0
call back block address (base+48)->00000000

boatCheck2 DBE created at: 08B541F0
call back block address (base+48)->00000000

pre-set DBE callback functions
Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
Allocated Objects: 1


post-set DBE callback functions
Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
   LE->1->08B50DA0->08B52FA0
   LE->2->08B541D0->08B55AC0
Allocated Objects: 3


boatCheck DBE after callback set: 08B50DC0
call back block address (base+48)->08B55AC0

boatCheck2 DBE after callback set: 08B541F0
call back block address (base+48)->08B52FA0

pre-block
Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
   LE->1->08B50DA0->08B52FA0
   LE->2->08B541D0->08B55AC0
Allocated Objects: 3

Dialog Box Closed

Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
   LE->1->08B50DA0->08B52FA0
   LE->2->08B541D0->08B55AC0
Allocated Objects: 3


DB destroyed:
Allocation List (cnt->list element address->object address):
   LE->0->08B50DA0->08B52FA0
   LE->1->08B541D0->08B55AC0
Allocated Objects: 2


Dump Allocated Objects
   LE->0->08B50DA0->08B52FA0
   content:
08B507F0
0C276BC8


   LE->1->08B541D0->08B55AC0
   content:
08B507F0
0C276BC8


Allocated Objects: 2

Re: Memory Magic Mystery
Mathias Mamsch - Tue Jan 17 05:33:49 EST 2017

ContextPtr: 08B507F0

Start
Allocation List (cnt->list element address->object address):
Allocated Objects: 0

DB created at: 08B50550

Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
Allocated Objects: 1

toBuild callback function address: 0C276BC8

boatCheck DBE created at: 08B50DC0
call back block address (base+48)->00000000

boatCheck2 DBE created at: 08B541F0
call back block address (base+48)->00000000

pre-set DBE callback functions
Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
Allocated Objects: 1


post-set DBE callback functions
Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
   LE->1->08B50DA0->08B52FA0
   LE->2->08B541D0->08B55AC0
Allocated Objects: 3


boatCheck DBE after callback set: 08B50DC0
call back block address (base+48)->08B55AC0

boatCheck2 DBE after callback set: 08B541F0
call back block address (base+48)->08B52FA0

pre-block
Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
   LE->1->08B50DA0->08B52FA0
   LE->2->08B541D0->08B55AC0
Allocated Objects: 3

Dialog Box Closed

Allocation List (cnt->list element address->object address):
   LE->0->08B55A60->08B50550
   LE->1->08B50DA0->08B52FA0
   LE->2->08B541D0->08B55AC0
Allocated Objects: 3


DB destroyed:
Allocation List (cnt->list element address->object address):
   LE->0->08B50DA0->08B52FA0
   LE->1->08B541D0->08B55AC0
Allocated Objects: 2


Dump Allocated Objects
   LE->0->08B50DA0->08B52FA0
   content:
08B507F0
0C276BC8


   LE->1->08B541D0->08B55AC0
   content:
08B507F0
0C276BC8


Allocated Objects: 2

You got two addresses inside the allocation list items. One is the address of the object. The second one is actually a function ptr to the internal destructor function of the data types. You can do something nice with it: 

- You can derive the data type of the allocated object from them (same type has same destructor, hopefully all types have different destructors)

Once you know the data type (e.g. set call back), you can choose to ignore this leak in your statistics to reduce noise. 

The second thing is, that you do not need to hunt down your leaks. You can make DOORS tell you exactly where in your code you leaked the object. For this you need to hook the constructor and destructor functions of a type. Inside the constructor you put a dxlHere() in a skip (key = address of the object). Inside the destructor you remove it from the skip list. After your code has run through, the skip list will contain the leaked objects with the dxlHere of the constructor, telling you exactly where you leaked the object. If you need some example code, just post (its already somewhere on the forum). 

Regarding the experiments on cleaning a data type up, you can use an eval_ (the good thing about eval_ is that it will clean up everything after the run). I think it is not safe to use a different destructor to clean up an object. I do not know how the work is divided between the internal destructor function (the one from the allocations list) and the DXL perm. This you would need to inspect using your favourite disassembler. If you find a DXL destructor for some type that does nothing but call the internal destructor function, you may have something you can use.

Regards, Mathias

Re: Memory Magic Mystery
GothSloth - Thu Feb 09 12:35:14 EST 2017

I found the following addresses for DOORS 9.5.2.1. I image these will be different for every release. A couple things that surprised me:

• IPC and Regexp have the same destructor address.
• DxlObjects don't make an appearance on the allocation list although Skips do.

0x00677014, "DB"
0x0075A12F, "ClipboardLock"
0x00B99DCD, "DbPropertiesCache"
0x00BB7DC6, "ModuleVersion"
0x00DF3BBC, "IPC"
0x00DF3BBC, "Regexp2"
0x00EA43DA, "Stat"
0x00FFB542, "OleAutoArgs"
0x02271328, "Array"
0x02272ABE, "Buffer"
0x02277EAC, "Skip"

NOT FOUND for type Column
NOT FOUND for type DxlObject
NOT FOUND for type ModName_
NOT FOUND for type Trigger
NOT FOUND for type ViewDef

For anyone with a bad case on insomnia I've attached the code for associating the destructor addresses with types. It appears the two objects left lingering from the dialog box are Stat objects or at least something that shares the same destructor.

Thanks for the pointer on the hook method. I found this post https://www.ibm.com/developerworks/community/forums/html/topic?id=77777777-0000-0000-0000-000014657796&ps=25 that discusses the approach and shows how to overload the constructors and destructors. I think I get how to implement the rest but if there is another post out there I would appreciate it.

Re: Memory Magic Mystery
Mathias Mamsch - Fri Feb 10 09:49:52 EST 2017

Nice. I never noticed, that DXL Object did not generate allocation entries ... Does this mean, that all DXL Objects leak permanently? 

Regarding the "hook" method, to find the leaks, what you need to do is something like this: 

Skip allocatedObjects = create(); 

void addAllocatedObject(int ad, string type) {
    // put in some checks, use the type to get some nicer infos
    put(allocatedObjects, ad, dxlHere(), true); 
}

void removeAllocatedObject(int ad, string type) {
    // put in some checks, if you try to delete an item, that you already deleted, etc. 
    delete(allocatedObjects, ad);
}

void printLeaks () {
        print "\n"; 
        string trace; for trace in allocatedObjects do {
            print "You leaked something here: " trace "\n\n"; 
        }
}

// Save access to Skip destructors/constructors
Skip create_old() { Skip sk= create(); return sk; }
void delete_old(Skip sk) { delete sk; }

// Override Skip constructors / destructors
Skip create() { Skip sk=create_old(); addAllocatedObject( (addr_ sk) int, "Skip");  return sk; }
void delete(Skip &sk) { Skip s = sk; removeAllocatedObject( (addr_ s) int, "Skip");  delete_old sk; }


// ***** Main code *****
Skip sk = create();  // Leak here, Line 30
Skip sk2 = create(); delete sk2;

printLeaks(); 

Running this code should give you something like: 

You leaked something here: <D:\temp\leaks.dxl:5>
<D:\temp\leaks.dxl:25>

<D:\temp\leaks.dxl:30>

Which tells you the exact location, where you allocated the leaked item. Now combine that with the allocations list. Walking through the allocations list at the end of your program, finding the entries that do NOT show up inside your skip list, you will find, if you forgot to hook a constructor / destructor pair. 

Hope this helps, regards, Mathias